package fetcher

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"time"

	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
)

var rateLimiter = time.Tick(10 * time.Millisecond)	// 请求频率

var userAgents = []string {
	"Mozilla/5.0(Macintosh;U;IntelMacOSX10_6_8;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50",
	"Mozilla/5.0(Windows;U;WindowsNT6.1;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50",
	"Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Trident/5.0)",	// IE9
	"Mozilla/4.0(compatible;MSIE8.0;WindowsNT6.0;Trident/4.0)",	// IE8
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT6.0)",	// IE7
	"Mozilla/4.0(compatible;MSIE6.0;WindowsNT5.1)",	// IE6
	"Mozilla/5.0(Macintosh;IntelMacOSX10.6;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
	"Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
	"Opera/9.80(Macintosh;IntelMacOSX10.6.8;U;en)Presto/2.8.131Version/11.11",
	"Opera/9.80(WindowsNT6.1;U;en)Presto/2.8.131Version/11.11",
	"Mozilla/5.0(Macintosh;IntelMacOSX10_7_0)AppleWebKit/535.11(KHTML,likeGecko)Chrome/17.0.963.56Safari/535.11",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Maxthon2.0)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TencentTraveler4.0)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TheWorld)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;360SE)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;AvantBrowser)",
	"Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1)",
	"Mozilla/5.0(iPhone;U;CPUiPhoneOS4_3_3likeMacOSX;en-us)AppleWebKit/533.17.9(KHTML,likeGecko)Version/5.0.2Mobile/8J2Safari/6533.18.5",
	"Mozilla/5.0(iPod;U;CPUiPhoneOS4_3_3likeMacOSX;en-us)AppleWebKit/533.17.9(KHTML,likeGecko)Version/5.0.2Mobile/8J2Safari/6533.18.5",
	"Mozilla/5.0(iPad;U;CPUOS4_3_3likeMacOSX;en-us)AppleWebKit/533.17.9(KHTML,likeGecko)Version/5.0.2Mobile/8J2Safari/6533.18.5",
	"Mozilla/5.0(Linux;U;Android2.3.7;en-us;NexusOneBuild/FRF91)AppleWebKit/533.1(KHTML,likeGecko)Version/4.0MobileSafari/533.1",
	"MQQBrowser/26Mozilla/5.0(Linux;U;Android2.3.7;zh-cn;MB200Build/GRJ22;CyanogenMod-7)AppleWebKit/533.1(KHTML,likeGecko)Version/4.0MobileSafari/533.1",
	"Opera/9.80(Android2.3.4;Linux;OperaMobi/build-1107180945;U;en-GB)Presto/2.8.149Version/11.10",
	"Mozilla/5.0(Linux;U;Android3.0;en-us;XoomBuild/HRI39)AppleWebKit/534.13(KHTML,likeGecko)Version/4.0Safari/534.13",
	"Mozilla/5.0(BlackBerry;U;BlackBerry9800;en)AppleWebKit/534.1+(KHTML,likeGecko)Version/6.0.0.337MobileSafari/534.1+",
	"Mozilla/5.0(hp-tablet;Linux;hpwOS/3.0.0;U;en-US)AppleWebKit/534.6(KHTML,likeGecko)wOSBrowser/233.70Safari/534.6TouchPad/1.0",
	"Mozilla/4.0(compatible;MSIE6.0;)Opera/UCWEB7.0.2.37/28/999",
}

// 抓取指定url的内容
func Fetch(url string) ([]byte, error) {
	<- rateLimiter
	// log.Printf("Fetch: %s\n", url)
	request, e := http.NewRequest(http.MethodGet, url, nil)
	if e != nil {
		return nil ,e
	}
	request.Header.Add("User-Agent", userAgents[rand.Intn(len(userAgents))])

	resp, err := http.DefaultClient.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	// 获取网页编码
	encoding := determineEncoding(bodyReader)
	// 将网页编码转成UTF8
	utf8Reader := transform.NewReader(bodyReader, encoding.NewDecoder())

	return ioutil.ReadAll(utf8Reader)
}

// 获取网页的编码
func determineEncoding(r *bufio.Reader) encoding.Encoding {
	bytes, err := r.Peek(1024)
	if err != nil {
		log.Printf("Fetcher get encoding error: %v", err)
		return unicode.UTF8
	}

	encoding, _, _ := charset.DetermineEncoding(bytes, "")
	return encoding
}
